<!doctype html>


<html>
<head>
  <link rel="shortcut icon" href="static/images/favicon.ico" type="image/x-icon">
  <title>Tokenizer_ (Closure Library API Documentation - JavaScript)</title>
  <link rel="stylesheet" href="static/css/base.css">
  <link rel="stylesheet" href="static/css/doc.css">
  <link rel="stylesheet" href="static/css/sidetree.css">
  <link rel="stylesheet" href="static/css/prettify.css">

  <script>
     var _staticFilePath = "static/";
     var _typeTreeName = "goog";
     var _fileTreeName = "Source";
  </script>

  <script src="static/js/doc.js">
  </script>

  <script type="text/javascript" src="//www.google.com/jsapi"></script>
  <script type="text/javascript">
     google.load("visualization", "1", {packages:["orgchart"]});
     var _loadingVisualizations = true;
  </script>

  <meta charset="utf8">
</head>

<body onload="grokdoc.onLoad();">

<div id="header">
  <div class="g-section g-tpl-50-50 g-split">
    <div class="g-unit g-first">
      <a id="logo" href="index.html">Closure Library API Documentation</a>
    </div>

    <div class="g-unit">
      <div class="g-c">
        <strong>Go to class or file:</strong>
        <input type="text" id="ac">
      </div>
    </div>
  </div>
</div>





<div class="colmask rightmenu">
<div class="colleft">
    <div class="col1">
      <!-- Column 1 start -->

<div id="title">
        <span class="fn">proto2.TextFormatSerializer.Tokenizer_</span>
        <span>Extends</span>
        <span class="fn-name">
          
        </span>
</div>

<div class="g-section g-tpl-75-25">
  <div class="g-unit g-first" id="description">
    Helper class for tokenizing the text format.

  </div>


        <div class="g-unit" id="useful-links">
          <div class="title">Useful links</div>
          <ol>
            <li><a href="<a href="http://goo.gl/QDmDr">http://goo.gl/QDmDr</a>">Also See:</a></li>
            <li><a href="local_closure_goog_proto2_textformatserializer.js.source.html"><span class='source-code-link'>Source Code</span></a></li>
            <li><a href="http://code.google.com/p/closure-library/source/browse/local/closure/goog/proto2/textformatserializer.js">Git</a></li>
          </ol>
        </div>
</div>






<h2 class="g-first">Inheritance</h2>
<div class='gviz-org-chart-container goog-inline-block'>
<table class='gviz-org-chart' style='display:none'>
  <tr><td>goog.proto2.TextFormatSerializer.Tokenizer_</td><td></td></tr>
</table>
</div>
<div class='gviz-org-chart-container goog-inline-block'>
<table class='gviz-org-chart' style='display:none'>
</table>
</div>




  <h2>Constructor</h2>
      <div class="fn-constructor">
    goog.proto2.TextFormatSerializer.Tokenizer_(<span><span class="arg">data</span>,&nbsp;<span class="arg">opt_ignoreWhitespace</span></span>)
  </div>

   <h3>Parameters</h3>






<table class="horiz-rule">
     
   <tr class="even">
     <td>
        <span class="entryName">data</span>
        : <span class="type"><a href="https://developer.mozilla.org/en/Core_JavaScript_1.5_Reference/Global_Objects/String">string</a></span>
        <div class="entryOverview">The string data to tokenize.</div>
     </td>
   </tr>
     
   <tr class="odd">
     <td>
        <span class="entryName">opt_ignoreWhitespace</span>
        : <span class="type">boolean=</span>
        <div class="entryOverview">If true, whitespace tokens will not
   be reported by the tokenizer.</div>
     </td>
   </tr>
  </table>


<h2>Instance Methods</h2>


  <div class="legend">
        <span class="key publickey"></span><span>Public</span>
        <span class="key protectedkey"></span><span>Protected</span>
        <span class="key privatekey"></span><span>Private</span>
  </div>







<div>
       <div class="desc">
         Defined in
         <span class="fn-name">
            <a href="class_goog_proto2_TextFormatSerializer_Tokenizer_.html">
              goog.proto2.TextFormatSerializer.Tokenizer_
            </a>
         </span>
       </div>






<div class="section">
  <table class="horiz-rule">


     <tr class="even entry public">
       <td class="access"></td>






  <td>
    <a name="goog.proto2.TextFormatSerializer.Tokenizer_.prototype.getCurrent"></a>


     <div class="arg">
       <img align="left" src="static/images/blank.gif">

        <span class="entryName">getCurrent<span class="args">()</span>
        </span>
        &#8658; <span class="type">goog.proto2.TextFormatSerializer.Tokenizer_.Token</span>
      </div>


     <div class="entryOverview">
       <span class='nodesc'>No description.</span>
     </div>


    <! -- Method details -->
    <div class="entryDetails">

   
      <div class="detailsSection">
        <b>Returns:</b>&nbsp;<span class="type">goog.proto2.TextFormatSerializer.Tokenizer_.Token</span>&nbsp;
            The current
    token.
      </div>
  
    </div>
   
  </td>


  <td class="view-code">
     <a href="local_closure_goog_proto2_textformatserializer.js.source.html#line446">code &raquo;</a>
  </td>
     </tr>


     <tr class="odd entry public">
       <td class="access"></td>






  <td>
    <a name="goog.proto2.TextFormatSerializer.Tokenizer_.prototype.next"></a>


     <div class="arg">
       <img align="left" src="static/images/blank.gif">

        <span class="entryName">next<span class="args">()</span>
        </span>
        &#8658; <span class="type"><a href="https://developer.mozilla.org/en/Core_JavaScript_1.5_Reference/Global_Objects/Boolean">boolean</a></span>
      </div>


     <div class="entryOverview">
       Advances to the next token.

     </div>


    <! -- Method details -->
    <div class="entryDetails">

   
      <div class="detailsSection">
        <b>Returns:</b>&nbsp;<span class="type"><a href="https://developer.mozilla.org/en/Core_JavaScript_1.5_Reference/Global_Objects/Boolean">boolean</a></span>&nbsp;
            True if a valid token was found, false if the end was
   reached or no valid token was found.
      </div>
  
    </div>
   
  </td>


  <td class="view-code">
     <a href="local_closure_goog_proto2_textformatserializer.js.source.html#line480">code &raquo;</a>
  </td>
     </tr>


     <tr class="even entry private">
       <td class="access"></td>






  <td>
    <a name="goog.proto2.TextFormatSerializer.Tokenizer_.prototype.nextInternal_"></a>


     <div class="arg">
       <img align="left" src="static/images/blank.gif">

        <span class="entryName">nextInternal_<span class="args">()</span>
        </span>
        &#8658; <span class="type"><a href="https://developer.mozilla.org/en/Core_JavaScript_1.5_Reference/Global_Objects/Boolean">boolean</a></span>
      </div>


     <div class="entryOverview">
       Internal method for determining the next token.

     </div>


    <! -- Method details -->
    <div class="entryDetails">

   
      <div class="detailsSection">
        <b>Returns:</b>&nbsp;<span class="type"><a href="https://developer.mozilla.org/en/Core_JavaScript_1.5_Reference/Global_Objects/Boolean">boolean</a></span>&nbsp;
            True if a next token was found, false otherwise.
      </div>
  
    </div>
   
  </td>


  <td class="view-code">
     <a href="local_closure_goog_proto2_textformatserializer.js.source.html#line505">code &raquo;</a>
  </td>
     </tr>


  </table>
</div>
</div>




<h2>Instance Properties</h2>







<div>
       <div class="desc">
         Defined in
         <span class="fn-name">
            <a href="class_goog_proto2_TextFormatSerializer_Tokenizer_.html">
              goog.proto2.TextFormatSerializer.Tokenizer_
            </a>
         </span>
       </div>






<div class="section">
  <table class="horiz-rule">


     <tr class="even entry private">
       <td class="access"></td>





  <a name="goog.proto2.TextFormatSerializer.Tokenizer_.prototype.currentData_"></a>

  <td>


     <div class="arg">
        <img align="left" src="static/images/blank.gif">

        <span class="entryName">currentData_</span>
        : 
     </div>


     <div class="entryOverview">
       The data string starting at the current index.

     </div>

  </td>


  <td class="view-code">
     <a href="local_closure_goog_proto2_textformatserializer.js.source.html#line421">Code &raquo;</a>
  </td>
     </tr>


     <tr class="odd entry private">
       <td class="access"></td>





  <a name="goog.proto2.TextFormatSerializer.Tokenizer_.prototype.current_"></a>

  <td>


     <div class="arg">
        <img align="left" src="static/images/blank.gif">

        <span class="entryName">current_</span>
        : 
     </div>


     <div class="entryOverview">
       The current token type.

     </div>

  </td>


  <td class="view-code">
     <a href="local_closure_goog_proto2_textformatserializer.js.source.html#line428">Code &raquo;</a>
  </td>
     </tr>


     <tr class="even entry private">
       <td class="access"></td>





  <a name="goog.proto2.TextFormatSerializer.Tokenizer_.prototype.data_"></a>

  <td>


     <div class="arg">
        <img align="left" src="static/images/blank.gif">

        <span class="entryName">data_</span>
        : 
     </div>


     <div class="entryOverview">
       The data being tokenized.

     </div>

  </td>


  <td class="view-code">
     <a href="local_closure_goog_proto2_textformatserializer.js.source.html#line407">Code &raquo;</a>
  </td>
     </tr>


     <tr class="odd entry private">
       <td class="access"></td>





  <a name="goog.proto2.TextFormatSerializer.Tokenizer_.prototype.ignoreWhitespace_"></a>

  <td>


     <div class="arg">
        <img align="left" src="static/images/blank.gif">

        <span class="entryName">ignoreWhitespace_</span>
        : 
     </div>


     <div class="entryOverview">
       Whether to skip whitespace tokens on output.

     </div>

  </td>


  <td class="view-code">
     <a href="local_closure_goog_proto2_textformatserializer.js.source.html#line400">Code &raquo;</a>
  </td>
     </tr>


     <tr class="even entry private">
       <td class="access"></td>





  <a name="goog.proto2.TextFormatSerializer.Tokenizer_.prototype.index_"></a>

  <td>


     <div class="arg">
        <img align="left" src="static/images/blank.gif">

        <span class="entryName">index_</span>
        : 
     </div>


     <div class="entryOverview">
       The current index in the data.

     </div>

  </td>


  <td class="view-code">
     <a href="local_closure_goog_proto2_textformatserializer.js.source.html#line414">Code &raquo;</a>
  </td>
     </tr>


  </table>
</div>
</div>










<div class="section">
  <table class="horiz-rule">


  </table>
</div>



<h2>Static Properties</h2>






<div class="section">
  <table class="horiz-rule">


     <tr class="even entry public">
       <td class="access"></td>





  <a name="goog.proto2.TextFormatSerializer.Tokenizer_.Token"></a>

  <td>


     <div class="arg">
        <img align="left" src="static/images/blank.gif">

        <span class="entryNamespace">goog.proto2.TextFormatSerializer.Tokenizer_.</span><span class="entryName">Token</span>
        : 
     </div>


     <div class="entryOverview">
       <span class='nodesc'>No description.</span>
     </div>

  </td>


  <td class="view-code">
     <a href="local_closure_goog_proto2_textformatserializer.js.source.html#line439">Code &raquo;</a>
  </td>
     </tr>


  </table>
</div>



<h2>Enumerations</h2>






<div class="section">
  <table class="horiz-rule">


     <tr class="even entry public">
       <td class="access"></td>





  <a name="goog.proto2.TextFormatSerializer.Tokenizer_.TokenTypes"></a>

  <td>


     <div class="arg">
        <img align="left" src="static/images/blank.gif">

        <span class="entryNamespace">goog.proto2.TextFormatSerializer.Tokenizer_.</span><span class="entryName">TokenTypes</span>
        : 
     </div>


     <div class="entryOverview">
       An enumeration of all the token types.

     </div>

     <div class="entryDetails">
       <div class="detailsSection">
          <b>Constants:</b>
          <table class="horiz-rule">
            <tbody>
              <tr><td>
                <span class="entryName">
                  CLOSE_BRACE
                </span>
                <div class="entryOverview">
                  <span class='nodesc'>No description.</span>
                </div>
             </td></tr>
              <tr><td>
                <span class="entryName">
                  CLOSE_LIST
                </span>
                <div class="entryOverview">
                  <span class='nodesc'>No description.</span>
                </div>
             </td></tr>
              <tr><td>
                <span class="entryName">
                  CLOSE_TAG
                </span>
                <div class="entryOverview">
                  <span class='nodesc'>No description.</span>
                </div>
             </td></tr>
              <tr><td>
                <span class="entryName">
                  COLON
                </span>
                <div class="entryOverview">
                  <span class='nodesc'>No description.</span>
                </div>
             </td></tr>
              <tr><td>
                <span class="entryName">
                  COMMA
                </span>
                <div class="entryOverview">
                  <span class='nodesc'>No description.</span>
                </div>
             </td></tr>
              <tr><td>
                <span class="entryName">
                  COMMENT
                </span>
                <div class="entryOverview">
                  <span class='nodesc'>No description.</span>
                </div>
             </td></tr>
              <tr><td>
                <span class="entryName">
                  END
                </span>
                <div class="entryOverview">
                  <span class='nodesc'>No description.</span>
                </div>
             </td></tr>
              <tr><td>
                <span class="entryName">
                  IDENTIFIER
                </span>
                <div class="entryOverview">
                  <span class='nodesc'>No description.</span>
                </div>
             </td></tr>
              <tr><td>
                <span class="entryName">
                  NUMBER
                </span>
                <div class="entryOverview">
                  <span class='nodesc'>No description.</span>
                </div>
             </td></tr>
              <tr><td>
                <span class="entryName">
                  OPEN_BRACE
                </span>
                <div class="entryOverview">
                  <span class='nodesc'>No description.</span>
                </div>
             </td></tr>
              <tr><td>
                <span class="entryName">
                  OPEN_LIST
                </span>
                <div class="entryOverview">
                  <span class='nodesc'>No description.</span>
                </div>
             </td></tr>
              <tr><td>
                <span class="entryName">
                  OPEN_TAG
                </span>
                <div class="entryOverview">
                  <span class='nodesc'>No description.</span>
                </div>
             </td></tr>
              <tr><td>
                <span class="entryName">
                  SEMI
                </span>
                <div class="entryOverview">
                  <span class='nodesc'>No description.</span>
                </div>
             </td></tr>
              <tr><td>
                <span class="entryName">
                  STRING
                </span>
                <div class="entryOverview">
                  <span class='nodesc'>No description.</span>
                </div>
             </td></tr>
              <tr><td>
                <span class="entryName">
                  WHITESPACE
                </span>
                <div class="entryOverview">
                  <span class='nodesc'>No description.</span>
                </div>
             </td></tr>
           </tbody>
         </table>
       </div>
     </div>

  </td>


  <td class="view-code">
     <a href="local_closure_goog_proto2_textformatserializer.js.source.html#line455">Code &raquo;</a>
  </td>
     </tr>


  </table>
</div>
      <!-- Column 1 end -->
    </div>

        <div class="col2">
          <!-- Column 2 start -->
          <div class="col2-c">
            <h2 id="ref-head">Package proto2.TextFormatSerializer</h2>
            <div id="localView"></div>
          </div>

          <div class="col2-c">
            <h2 id="ref-head">Package Reference</h2>
            <div id="sideTypeIndex" rootPath="" current="proto2.TextFormatSerializer.Tokenizer_"></div>
          </div>
          <!-- Column 2 end -->
        </div>
</div>
</div>

</body>
</html>
